#pragma once
#include "MDPSolver.h"
#include "Tiles.h"

/*
	This planner literaly runs a sarsa agent with tile coding in the simulator to learn a policy
*/

namespace planning
{

//some constants
#define MEMORY_SIZE 10000                       //number of possible features 
#define	M_PI	3.141597
const int  MAX_NONZERO_TRACES=1000;



class SarsaPlanner :
	public MDPSolver
{
public:
	SarsaPlanner(MREAgent* a, TFGenerator* tf, RFGeneralizer* rf); 
	virtual ~SarsaPlanner(void);

	virtual void operator ()(); 
	virtual Action getBestAction(Observation state); 
	virtual void	solveModel(Observation currentState); 
	virtual double getStateValue(const Observation o); 
	virtual double getStateValueFast(const Observation o); 


	
Action agent_start(Observation o,bool greedy = false);
Action agent_step(double r, Observation o, bool greedy=false);
void agent_end(double r);
void selectEpsilonGreedyAction(Observation o, Action& action, bool greedy=false);
void updateTraces();
void updateWeights(double delta);
void computeActionValues() ;
void computeActionValues(Action a) ;
void computeActiveFeatures(Observation o);
int argmax(double* QSA);
void SetTrace(int f, double new_trace_value);
void ClearTrace(int f)       ;
void ClearExistentTrace(int f, int loc);
void DecayTraces(double decay_rate);
void IncreaseMinTrace();
void makeRandomState(Observation st); 
template < typename T >
T ** Allocate2DArray( int nRows, int nCols)
{
    T **ppi;
    T *pool;
    T *curPtr;
    //(step 1) allocate memory for array of elements of column

    ppi = new T*[nRows];

    //(step 2) allocate memory for array of elements of each row
    pool = new T [nRows * nCols];

    // Now point the pointers in the right place
    curPtr = pool;
    for( int i = 0; i < nRows; i++)
    {
        *(ppi + i) = curPtr;
         curPtr += nCols;
    }
    return ppi;
}

template < typename T >
void Free2DArray(T** Array)
{
    delete [] *Array;
    delete [] Array;
}
	









public: 
	double* tileWidth; 

	// Global RL variables:
	int NUM_ACTIONS;                         //number of available actions   
	int NUM_TILINGS; 						  //number of tiling grids	
	int NUM_OBSERVATIONS;                     //dimension of observation


	double* QSA;								// action values
	double* weights;							// feature weights
	double* traces;                             // eligibility traces
	int** activeFeatures;           // sets of features, one set per action

	// Standard RL parameters:
	double epsilon;                    // probability of random action
	double alpha;                      // step size parameter
	double lambda;                     // trace-decay parameters
	double RLgamma;                // discount-rate parameters

	Action oldAction;						//action selected on previous time step
	Action newAction;						//action selected on current time step

	taskspec_t *ts;					//task spec


	//sutton 
	int nonzero_traces[MAX_NONZERO_TRACES];
	int* nonzero_traces_inverse;

	double minimum_trace ;
	int num_nonzero_traces;
};
}//namespace